{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/177.0.2656.135 Safari/537.36'}\n",
      "https://www.suruga-ya.jp/search?category=&search_word=Doraemon&adult_s=1&is_marketplace=0&page=1\n",
      "24\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "23\n",
      "24\n",
      "suruga当前共爬取1页数据\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<myItem.myItem at 0x207a7cc1fa0>,\n",
       " <myItem.myItem at 0x207a7f19760>,\n",
       " <myItem.myItem at 0x207a7f19610>,\n",
       " <myItem.myItem at 0x207a7f19790>,\n",
       " <myItem.myItem at 0x207a7f19730>,\n",
       " <myItem.myItem at 0x207a7f19820>,\n",
       " <myItem.myItem at 0x207a7f19880>,\n",
       " <myItem.myItem at 0x207a7f198e0>,\n",
       " <myItem.myItem at 0x207a7f19940>,\n",
       " <myItem.myItem at 0x207a7f199a0>,\n",
       " <myItem.myItem at 0x207a7f19a00>,\n",
       " <myItem.myItem at 0x207a7f19a60>,\n",
       " <myItem.myItem at 0x207a7f19ac0>,\n",
       " <myItem.myItem at 0x207a7f19b20>,\n",
       " <myItem.myItem at 0x207a7f19b80>,\n",
       " <myItem.myItem at 0x207a7f19be0>,\n",
       " <myItem.myItem at 0x207a7f19c40>,\n",
       " <myItem.myItem at 0x207a7f19ca0>,\n",
       " <myItem.myItem at 0x207a7f19d00>,\n",
       " <myItem.myItem at 0x207a7f19d60>,\n",
       " <myItem.myItem at 0x207a7f19dc0>,\n",
       " <myItem.myItem at 0x207a7f19e20>,\n",
       " <myItem.myItem at 0x207a7f19f10>,\n",
       " <myItem.myItem at 0x207a7f19fd0>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import traceback\n",
    "import random\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from bs4 import element\n",
    "import datetime\n",
    "import chardet\n",
    "import Ipynb_importer\n",
    "from myItem import myItem\n",
    "from compareItems import compareItems\n",
    "from sendMail import sendMail\n",
    "\n",
    "\n",
    "class newNotice(object):\n",
    "    def __init__(self):\n",
    "        self.url = \"\"\n",
    "        self.headers = {}\n",
    "        self.notice=''\n",
    "        \n",
    "    def setHeaders(self):\n",
    "        headStr='Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/'\n",
    "        tailStr=' Safari/537.36'\n",
    "        num1=random.randint(1,255)\n",
    "        num3=random.randint(1,9999)\n",
    "        num4=random.randint(1,255)\n",
    "        self.headers = {'user-agent':headStr+str(num1)+'.0.'+str(num3)+'.'+str(num4)+tailStr}\n",
    "        print(str(self.headers))\n",
    "        \n",
    "    def setSearchWord(self,searchWord):\n",
    "        url = \"https://www.suruga-ya.jp/search?category=\"\n",
    "        self.searchWord=\"&search_word=\"+searchWord\n",
    "        self.baseURL=url+self.searchWord+\"&adult_s=1&is_marketplace=0\"\n",
    "        \n",
    "    def setPage(self,page):\n",
    "        self.page='&page='+str(page)\n",
    "        self.url=self.baseURL+self.page\n",
    "        \n",
    "    def get_response(self,url):\n",
    "            print(url)\n",
    "            response = requests.get(url,headers=self.headers,timeout=10)\n",
    "            data = response.content\n",
    "            #print(data)\n",
    "            return data\n",
    "\n",
    "    def data_save(self,data):\n",
    "                with open('suruga.html','wb') as file:\n",
    "                    file.write(data)\n",
    "\n",
    "    def operationDelete(self,string):\n",
    "        string=list(string)\n",
    "        count=0\n",
    "        for i in range(len(string)):\n",
    "            if string[i] =='￥' or string[i]==',':\n",
    "                count=count+1\n",
    "                for j in range(i,len(string)-count):\n",
    "                    string[j]=string[j+1]\n",
    "        string=''.join(string[0:len(string)-count])\n",
    "        return string\n",
    "                \n",
    "    def parse_data(self,data):\n",
    "        productList=[]\n",
    "        count=0\n",
    "        soup = BeautifulSoup(data, 'html.parser')\n",
    "        for i in soup.findAll(name='p',attrs = {'class':'title'}):\n",
    "            pname=i.a.text[0:]\n",
    "            productURL = i.a['href']\n",
    "            if productURL[0]!='h': #可以利用地方店铺商品的网址不全的特性，来判断是否为地方店铺商品（需在myItem类中加一个字段用来存储，未实现）\n",
    "                productURL='https://www.suruga-ya.jp/'+productURL\n",
    "            #print(chardet.detect(pname.encode()))\n",
    "            #print(pname)\n",
    "            #print (productURL)\n",
    "            #print('\\n')\n",
    "            \n",
    "            item=myItem(None,None,None,None,None,None,None,datetime.datetime.now())\n",
    "            item.setPname(pname)\n",
    "            item.setProductURL(productURL)\n",
    "            productList.append(item)\n",
    "            \n",
    "        print(len(productList))\n",
    "        \n",
    "        for j in soup.findAll(name='div',attrs = {'class':'item_price'}):\n",
    "            if len(list(j.findAll(name='p',attrs = {'class':'price'})))==1:\n",
    "                price= self.operationDelete(list(j.findAll(name='span',attrs = {'class':'text-red fontS15'}))[0].strong.text[0:])\n",
    "                productList[count].setPrice(price)\n",
    "                count=count+1\n",
    "                '''for t1 in j.findAll(name='p',attrs = {'class':'mgnB5 mgnT5'}):\n",
    "                    for temp1 in t1.findAll(name='span',attrs = {'class':'text-red fontS15'}):\n",
    "                        price= self.operationDelete(temp1.strong.text[0:])\n",
    "                        #print(price)\n",
    "                        #print('\\n')\n",
    "                        productList[count].setPrice(price)\n",
    "                        count=count+1'''\n",
    "            else:\n",
    "                price= self.operationDelete(list(j.findAll(name='span',attrs = {'class':'text-red'}))[0].strong.text[0:])\n",
    "                productList[count].setPrice(price)\n",
    "                count=count+1\n",
    "                '''for t2 in j.findAll(name='p',attrs = {'class':'price_teika'}):\n",
    "                    for temp2 in t2.findAll(name='span',attrs = {'class':'text-red'}):\n",
    "                        price= self.operationDelete(temp2.strong.text[0:])\n",
    "                        #print(price)\n",
    "                        #print('\\n')\n",
    "                        print(count)\n",
    "                        print(\"aaaaa\")\n",
    "                        productList[count].setPrice(price)\n",
    "                        count=count+1'''\n",
    "            print(count)\n",
    "            \n",
    "        count=0\n",
    "        for k in soup.findAll(name='p',attrs = {'class':'thum'}):\n",
    "                img= k.a.img['src']\n",
    "                #print(img)\n",
    "                #print('\\n')\n",
    "            \n",
    "                productList[count].setDescription(img)\n",
    "                count=count+1\n",
    "        #for i in range(len(productList)):\n",
    "            #print(\"商品名称: \"+productList[i].getPname()+'\\n')\n",
    "            #print(\"价格: \"+str(productList[i].getPrice())+'\\n')\n",
    "            #print(\"商品网址: \"+productList[i].getProductURL()+'\\n')\n",
    "            #print(\"图片网址: \"+productList[i].getDescription()+'\\n')\n",
    "            #print(\"更新时间: \"+str(productList[i].getUpdateTime())+'\\n')\n",
    "            #print('\\n')\n",
    "        return productList\n",
    "    \n",
    "    def downloadImages(self,newProductList):\n",
    "        imgURL=''\n",
    "        #if os.path.exists(\"./tempImages\") != True:\n",
    "            #os.mkdir(\"./tempImages\")\n",
    "        \n",
    "        for i in range(len(newProductList)):\n",
    "            imgURL=newProductList[i].getDescription()\n",
    "            try:\n",
    "                img=requests.get(imgURL,headers=self.headers,timeout=10)\n",
    "                #将图片暂存到本地\n",
    "                #img_name=str(i)\n",
    "                #with open(\"./tempImages/\" + img_name + \".jpg\", \"wb\") as f:\n",
    "                    #f.write(img.content)\n",
    "                #print('writed')\n",
    "                #将本地图片转化为数据流赋值给newProductList中的实例对象\n",
    "                #with open(\"./tempImages/\" + img_name + \".jpg\", \"rb\") as finMySQL:\n",
    "                    #imgMySQL=finMySQL.read()\n",
    "                #newProductList[i].setProPic(imgMySQL)\n",
    "                #finMySQL.close()\n",
    "                \n",
    "                #os.remove(\"./tempImages/\" + img_name + \".jpg\")\n",
    "                #print('deleted')\n",
    "                \n",
    "                newProductList[i].setProPic(img.content)\n",
    "                \n",
    "            except:  #可以捕获所有异常\n",
    "                error = traceback.format_exc()\n",
    "                sendMailInstance=sendMail()\n",
    "                msg=sendMailInstance.sendError(error,'suruga图片爬取出错')\n",
    "                sendMailInstance.send(msg)\n",
    "                print(\"发送suruga图片爬取出错邮件\")\n",
    "                break\n",
    "            #time.sleep(0.1)\n",
    "        return newProductList\n",
    "            \n",
    "    def run(self,searchWord,pageNum):\n",
    "        self.setHeaders()\n",
    "        searchWord=searchWord\n",
    "        self.setSearchWord(searchWord)\n",
    "        newProductList=[]\n",
    "        try:\n",
    "            i=1\n",
    "            for i in range(1,pageNum):\n",
    "                self.setPage(i)\n",
    "                data=self.get_response(self.url)\n",
    "                #self.data_save(data)\n",
    "                productList=self.parse_data(data)\n",
    "                newProductList.extend(productList)\n",
    "                #if i%5==0:\n",
    "                    #self.setHeaders()\n",
    "                time.sleep(1)\n",
    "        except:  #可以捕获所有异常\n",
    "            error = traceback.format_exc()\n",
    "            sendMailInstance=sendMail()\n",
    "            msg=sendMailInstance.sendError(error,'suruga文本爬取出错')\n",
    "            sendMailInstance.send(msg)\n",
    "            print(\"发送suruga文本爬取出错邮件\")\n",
    "        finally:\n",
    "            #toMySQLInstance.close(conn)\n",
    "            print('suruga当前共爬取'+str(i)+'页数据')\n",
    "        return newProductList\n",
    "\n",
    "newNoticeInstance=newNotice()\n",
    "newNoticeInstance.run(\"+Doraemon\",2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
